package com.zhucx.utils;

import com.zhucx.core.enums.SelectTypeEnums;
import com.zhucx.model.PageRequest;
import org.apache.commons.lang3.ObjectUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import javax.net.ssl.*;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;

/**
 * jsoup tool
 *
 */
public class JsoupUtil {
    private  Logger logger = LoggerFactory.getLogger(JsoupUtil.class);
    // 连接失败最多尝试三次
    public  Integer times = 0;
    public  void main(String[] args) {
        String url = "https://machtalk.xiachufang.com/category/40076/";
        try {
            // 请求设置
            Connection conn = Jsoup.connect(url);
            conn.maxBodySize(0);    // 取消默认1M限制

            // 发出请求
            Document html = null;
                html = conn.get();

           Set<String> links =  findLinks(html);

           for(String set:links){
               System.out.println(set);
           }
//                List<Element> ael = html.select("a[href]");
//                for(Element el:ael){
//                    System.out.println(el.attr("abs:href"));
//                }
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
        }
    }


//    public static Document getJunshiByNotProxy(String requestUrl, Object lock) {
        public  Document load(PageRequest pageRequest,String reqType) {
        System.out.println("正在请求url：" + pageRequest.getUrl());
//        boolean isWallSwitch  = Boolean.valueOf(SPIDER_WALL_SWTICH);
//        if(isWallSwitch){
//            String proxyHost = SPIDER_WALL_URL ;
//            String proxyPort = SPIDER_WALL_PORT;
//            System.setProperty("http.proxyHost", proxyHost);
//            System.setProperty("http.proxyPort", proxyPort);
//            // 对https也开启代理
//            System.setProperty("https.proxyHost", proxyHost);
//            System.setProperty("https.proxyPort", proxyPort);
//        }

            String requestUrl = pageRequest.getUrl();
            Object lock = new Object();
        Random r = new Random();
        String[] ua = {
                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586",
                "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
                "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
                "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0" };
        int i = r.nextInt(14);
        String html = null;
        Document doc = null;

        try {
            trustAllHttpsCertificates();
            HttpsURLConnection.setDefaultHostnameVerifier(HttpsURLConnection.getDefaultHostnameVerifier());
            if("get".equals(reqType)){
                doc = Jsoup.connect(requestUrl).timeout(5000)
                        .ignoreContentType(true).userAgent(ua[i])
                        .header("referer", "https://baike.baidu.com")// 这个来源记得换..
                        .get();
            }else{
                doc = Jsoup.connect(requestUrl).timeout(5000)
                        .ignoreContentType(true).userAgent(ua[i])
                        .header("referer", "https://baike.baidu.com")// 这个来源记得换..
                        .post();
            }

            doc.location();


            // 记录没有请求到的url
            if (ObjectUtils.isEmpty(doc)) {
                synchronized (lock) {
                    System.out.println("没有请求到url....");
                    return null;
                }
            }
        } catch (Exception e) {
            // System.out.println("抓取失败了，当前线程：" +
            // Thread.currentThread().getName());
            synchronized (lock) {
                System.out.println("抓取失败了..." + e.getMessage());

                System.out.println("在次尝试....");
                times++;
                if (times < 3) {
//                    doc = load(requestUrl, lock);
                    doc = load(pageRequest,"post");
                }else{
                    return null;
                }
//				return doc;
            }
        }
        times = 0;
//		System.out.println("请求完毕");
        return doc;
    }



    /**
     * 加载页面
     *
     * @param pageRequest
     *
     * @return Document
     */
    public  Document load2(PageRequest pageRequest) {
        if (!UrlUtil.isUrl(pageRequest.getUrl())) {
            return null;
        }
        try {
            // 请求设置
            Connection conn = Jsoup.connect(pageRequest.getUrl());
            if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
                conn.data(pageRequest.getParamMap());
            }
            if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
                conn.cookies(pageRequest.getCookieMap());
            }
            if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
                conn.headers(pageRequest.getHeaderMap());
            }
            if (pageRequest.getUserAgent()!=null) {
                conn.userAgent(pageRequest.getUserAgent());
            }
            if (pageRequest.getReferrer() != null) {
                conn.referrer(pageRequest.getReferrer());
            }
//            conn.timeout(pageRequest.getTimeoutMillis());
            if (pageRequest.isValidateTLSCertificates()) {
                conn.sslSocketFactory(generateSSLSocketFactory());
            }
            conn.maxBodySize(0);    // 取消默认1M限制

            // 代理
            if (pageRequest.getProxy() != null) {
                conn.proxy(pageRequest.getProxy());
            }

            // 发出请求
            Document html = null;
            if (pageRequest.isIfPost()) {
                html = conn.post();
            } else {
                html = conn.get();
            }
            return html;
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
            return null;
        }
    }
    private  SSLSocketFactory generateSSLSocketFactory() {
        TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() {
            public X509Certificate[] getAcceptedIssuers() {
                return new X509Certificate[0];
            }
            public void checkClientTrusted(X509Certificate[] certs, String authType) {
            }
            public void checkServerTrusted(X509Certificate[] certs, String authType) {
            }
        } };
        try {
            SSLContext sslContext = SSLContext.getInstance("SSL");
            sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
            SSLSocketFactory result = sslContext.getSocketFactory();
            return result;
        } catch (KeyManagementException | NoSuchAlgorithmException e) {
            throw new RuntimeException("Failed to create a SSL socket factory", e);
        }
    }


    public  String loadPageSource(PageRequest pageRequest) {
        if (!UrlUtil.isUrl(pageRequest.getUrl())) {
            return null;
        }
        try {
            // 请求设置
            Connection conn = Jsoup.connect(pageRequest.getUrl());
            if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
                conn.data(pageRequest.getParamMap());
            }
            if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
                conn.cookies(pageRequest.getCookieMap());
            }
            if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
                conn.headers(pageRequest.getHeaderMap());
            }
            if (pageRequest.getUserAgent()!=null) {
                conn.userAgent(pageRequest.getUserAgent());
            }
            if (pageRequest.getReferrer() != null) {
                conn.referrer(pageRequest.getReferrer());
            }
            conn.timeout(pageRequest.getTimeoutMillis());
            if (pageRequest.isValidateTLSCertificates()) {
                conn.sslSocketFactory(generateSSLSocketFactory());
            }
            conn.maxBodySize(0);    // 取消默认1M限制

            // 代理
            if (pageRequest.getProxy() != null) {
                conn.proxy(pageRequest.getProxy());
            }

            conn.ignoreContentType(true);
            conn.method(pageRequest.isIfPost()?Connection.Method.POST:Connection.Method.GET);

            // 发出请求
            Connection.Response resp = conn.execute();
            String pageSource = resp.body();
            return pageSource;
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
            return null;
        }
    }

    /**
     * 抽取元素数据
     *
     * @param fieldElement
     * @param selectType
     * @param selectVal
     * @return String
     */
    public  String parseElement(Element fieldElement, SelectTypeEnums selectType, String selectVal) {
        String fieldElementOrigin = null;
        if (SelectTypeEnums.HTML == selectType) {
            fieldElementOrigin = fieldElement.html();
        } else if (SelectTypeEnums.VAL == selectType) {
            fieldElementOrigin = fieldElement.val();
        } else if (SelectTypeEnums.TEXT == selectType) {
            fieldElementOrigin = fieldElement.text();
        } else if (SelectTypeEnums.ATTR == selectType) {
            fieldElementOrigin = fieldElement.attr(selectVal);
        }  else if (SelectTypeEnums.HAS_CLASS == selectType) {
            fieldElementOrigin = String.valueOf(fieldElement.hasClass(selectVal));
        }  else {
            fieldElementOrigin = fieldElement.toString();
        }
        return fieldElementOrigin;
    }
    private  void trustAllHttpsCertificates() throws Exception {
        TrustManager[] trustAllCerts = new TrustManager[1];
        TrustManager tm = new miTM();
        trustAllCerts[0] = tm;
        SSLContext sc = SSLContext
                .getInstance("SSL");
        sc.init(null, trustAllCerts, null);
        javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc
                .getSocketFactory());
    }
    /**
     * 获取页面上所有超链接地址 （<a>标签的href值）
     *
     * @param html  页面文档
     * @return Set<String>
     */
    public  Set<String> findLinks(Element html) {

        if (html == null) {
            return null;
        }
        // element
        /**
         *
         * Elements resultSelect = html.select(tagName);	// 选择器方式
         * Element resultId = html.getElementById(tagName);	// 元素ID方式
         * Elements resultClass = html.getElementsByClass(tagName);	// ClassName方式
         * Elements resultTag = html.getElementsByTag(tagName);	// html标签方式 "body"
         *
         */
//        Document dos2 = Jsoup.parse(html.html());
        List<Element> hrefElements = html.select("a[href]");
        // 抽取数据
        Set<String> links = new HashSet<String>();
        if (hrefElements!=null && hrefElements.size() > 0) {
            for (Element item : hrefElements) {
                String href = item.attr("abs:href");    // href、abs:href
                if (UrlUtil.isUrl(href)) {
                    links.add(href);
                }
            }
        }
        //布隆过滤器过滤重复
        Set<String> filterLinks = new HashSet<String>();
        BloomFilter filter = new BloomFilter();
        int num = 0;
        for (String s : links) {
            if (!filter.contain(s)) {
                filterLinks.add(s);
                num++;
            }
        }

        return filterLinks;
    }

    /**
     * 获取页面上所有图片地址 （<a>标签的href值）
     *
     * @param html
     * @return Set<String>
     */
    public  Set<String> findImages(Document html) {

        Elements imgs = html.getElementsByTag("img");

        Set<String> images = new HashSet<String>();
        if (imgs!=null && imgs.size() > 0) {
            for (Element element: imgs) {
                String imgSrc = element.attr("abs:src");
                images.add(imgSrc);
            }
        }

        return images;
    }

     class miTM implements TrustManager,
            X509TrustManager {
        public X509Certificate[] getAcceptedIssuers() {
            return null;
        }

        public boolean isServerTrusted(
                X509Certificate[] certs) {
            return true;
        }

        public boolean isClientTrusted(
                X509Certificate[] certs) {
            return true;
        }

        public void checkServerTrusted(
                X509Certificate[] certs, String authType)
                throws java.security.cert.CertificateException {
            return;
        }

        public void checkClientTrusted(
                X509Certificate[] certs, String authType)
                throws java.security.cert.CertificateException {
            return;
        }
    }

}
